{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NHPP - Wavenet for Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NHPP (Near History Past Points) Wavenet was meant to test the information flow through the Wavenet Time-Series predictor. It splits the dataset into two separate parts: one containing the 'older' datapoints, and one containing the more recent ones. The older datapoints are ran through the Wavenet and its result is appended to the more recent datapoints. This was supposed to give the last linear layer the most important datapoints for the prediction (which were regarded as being the most recent ones) without the information they possess being twisted by Wavenet. \n",
    "\n",
    "It was found that this method works on some datasets but not on others, (very slightly) improving or diminishing the accuracy (though with low probability of this being due to chance)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nnkKq8DV3ThO"
   },
   "outputs": [],
   "source": [
    "# Importing modules\n",
    "\n",
    "import pandas as pd\n",
    "import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "import random\n",
    "from torch import nn, optim\n",
    "from torch.autograd import Variable\n",
    "from IPython.display import Audio\n",
    "from scipy.io import wavfile\n",
    "from collections import Counter\n",
    "from collections import defaultdict\n",
    "import math\n",
    "import copy\n",
    "import random\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "53FeW83gQ20S"
   },
   "outputs": [],
   "source": [
    "# Importing the data\n",
    "\n",
    "bench_file = 'predictionsBenchmark'\n",
    "tanke_bench = []\n",
    "train_targets_bench = []\n",
    "test_targets_bench = []\n",
    "df = pd.read_csv(bench_file + '.csv')\n",
    "months = []\n",
    "days = []\n",
    "weekdays = []\n",
    "index = 0\n",
    "value2index = defaultdict(int)\n",
    "file = 'extendedBikeData'\n",
    "for row in df.values:\n",
    "  if (file in row[4]):\n",
    "    \n",
    "    if (math.isnan(row[3]) == False):\n",
    "      tanke_bench.append(row[3])\n",
    "      test_targets_bench.append(row[2])\n",
    "      value2index[row[2]] = index\n",
    "      \n",
    "    elif(math.isnan(row[2]) == False):\n",
    "      \n",
    "      train_targets_bench.append(row[2])      \n",
    "      test_targets_bench.append(row[2])\n",
    "      value2index[row[2]] = index\n",
    "\n",
    "    index += 1\n",
    "    \n",
    "    year = int(row[1].split('-')[0])\n",
    "    month = int(row[1].split('-')[1])\n",
    "    day = int(row[1].split('-')[2].split(' ')[0])\n",
    "    date = datetime.date(year, month, day)\n",
    "    months.append(month)\n",
    "    days.append(day)\n",
    "    weekdays.append(date.weekday())\n",
    "#     print (date)\n",
    "tanke_bench = tanke_bench[0:360] # Filtering for the predictions for which we can compare (the last 30 have no targets)\n",
    "test_targets_bench = test_targets_bench[0:360] # Same here\n",
    "print (len(tanke_bench))\n",
    "train_targets_bench.extend(test_targets_bench)\n",
    "vals = train_targets_bench\n",
    "print (len(vals))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QoPqgOnPu9ny"
   },
   "outputs": [],
   "source": [
    "# Rescaling the data\n",
    "  \n",
    "class RMSELoss(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.mse = nn.MSELoss()\n",
    "        \n",
    "    def forward(self,yhat,y):\n",
    "        return torch.sqrt(self.mse(yhat,y))\n",
    "\n",
    "      \n",
    "      \n",
    "# Splitting dataset in train/test/valid datasets\n",
    "test_set_size = 360\n",
    "test_proportion = test_set_size/len(vals)\n",
    "validation_proportion = 0.2\n",
    "\n",
    "val_break = round(len(vals)*(1-(test_proportion + validation_proportion)))\n",
    "test_break = round(len(vals)*(1-test_proportion))\n",
    "\n",
    "values = vals[0 : val_break]\n",
    "values_valid = vals[val_break : test_break]\n",
    "values_test = vals[test_break:(len(vals))]\n",
    "      \n",
    "      \n",
    "      \n",
    "values_normed = torch.Tensor(vals)\n",
    "values = torch.Tensor(values)\n",
    "values_test = torch.Tensor(values_test)\n",
    "values_valid = torch.Tensor(values_valid)\n",
    "\n",
    "\n",
    "\n",
    "max_vals = max(vals)\n",
    "\n",
    "values_normed = values_normed/ max_vals\n",
    "values = values/max_vals\n",
    "values_test = values_test/max_vals\n",
    "values_valid = values_valid/max_vals\n",
    "\n",
    "\n",
    "loss_fct = RMSELoss()\n",
    "\n",
    "values_normed = values_normed.data.tolist()\n",
    "values_test = values_test.data.tolist()\n",
    "values = values.data.tolist()\n",
    "values_valid = values_valid.data.tolist()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JuVlC8OpWtGI"
   },
   "outputs": [],
   "source": [
    "\n",
    "# Function used to create the near history past points (nhpp for short)\n",
    "def past_points_collector(values, nhpp_nbr):\n",
    "  # Precomputing nhpp and additional features\n",
    "  # nhpp = near history past points (i.e. past points introduced as features)\n",
    "  # additional_features = days of the month (not one-hotted) + one-hot days of the week + one-hot months of the year\n",
    "\n",
    "  \n",
    "  if (additional_features == 50):\n",
    "  \n",
    "  \n",
    "    # history = torch.FloatTensor(values).cuda()\n",
    "    nhpp = []\n",
    "\n",
    "    days_oh = list(np.zeros(31))\n",
    "    days_oh[days[nhpp_nbr]] = 1\n",
    "    nhpp.extend(days_oh)\n",
    "\n",
    "    weekdays_oh = [0, 0, 0, 0, 0, 0, 0]\n",
    "    weekdays_oh[weekdays[nhpp_nbr]] = 1\n",
    "    nhpp.extend(weekdays_oh)\n",
    "\n",
    "    months_oh = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "    months_oh[months[nhpp_nbr]] = 1\n",
    "    nhpp.extend(months_oh)\n",
    "\n",
    "    nhpp.extend(values[0 : nhpp_nbr])\n",
    "\n",
    "\n",
    "    nhpp = torch.FloatTensor(nhpp).unsqueeze(1).unsqueeze(0).cuda()\n",
    "\n",
    "    for i in range (nhpp_nbr  + 1, len(values)):\n",
    "\n",
    "      nhpp_temp = []\n",
    "      days_oh = list(np.zeros(31))\n",
    "      days_oh[days[i]-1] = 1\n",
    "      nhpp_temp.extend(days_oh)\n",
    "\n",
    "      weekdays_oh = [0, 0, 0, 0, 0, 0, 0]\n",
    "      weekdays_oh[weekdays[i]] = 1\n",
    "      nhpp_temp.extend(weekdays_oh)\n",
    "\n",
    "      months_oh = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "      months_oh[months[nhpp_nbr]] = 1\n",
    "      nhpp_temp.extend(months_oh)\n",
    "      nhpp_temp.extend(values[i - nhpp_nbr : i])\n",
    "\n",
    "      nhpp_temp = torch.FloatTensor(nhpp_temp).unsqueeze(1).unsqueeze(0).cuda()\n",
    "      nhpp = torch.cat((nhpp, nhpp_temp), dim = 2)\n",
    "  \n",
    "  elif(additional_features == 0):\n",
    "    \n",
    "    nhpp = []\n",
    "    \n",
    "    if (nhpp_nbr > 0):\n",
    "      nhpp.extend(values[0 : nhpp_nbr])\n",
    "      nhpp = torch.FloatTensor(nhpp).unsqueeze(1).unsqueeze(0).cuda()\n",
    "      for i in range (nhpp_nbr + 1, len(values)):\n",
    "\n",
    "        nhpp_temp = []\n",
    "\n",
    "        nhpp_temp.extend(values[i - nhpp_nbr : i])\n",
    "        nhpp_temp = torch.FloatTensor(nhpp_temp).unsqueeze(1).unsqueeze(0).cuda()\n",
    "        \n",
    "        nhpp = torch.cat((nhpp, nhpp_temp), dim = 2)\n",
    "    \n",
    "    else:\n",
    "      nhpp = torch.FloatTensor(nhpp).unsqueeze(1).unsqueeze(0).cuda()\n",
    "      for i in range (0, len(values)):\n",
    "        nhpp_temp = torch.FloatTensor([]).unsqueeze(1).unsqueeze(0).cuda()\n",
    "        \n",
    "        nhpp = torch.cat((nhpp,nhpp_temp), dim = 2)\n",
    "      \n",
    "    \n",
    "    \n",
    "  elif(additional_features == 3): # additional_features not one-hotted. Not recommended\n",
    "    nhpp = [days[nhpp_nbr], weekdays[nhpp_nbr], months[nhpp_nbr]]\n",
    "    if (nhpp_nbr > 0):\n",
    "      nhpp.extend(values[0 : nhpp_nbr])\n",
    "      nhpp = torch.FloatTensor(nhpp).unsqueeze(1).unsqueeze(0).cuda()\n",
    "\n",
    "      for i in range (nhpp_nbr + 1, len(values)):\n",
    "        nhpp_temp = [days[i], weekdays[i], months[i]]\n",
    "        nhpp_temp.extend(values[i - nhpp_nbr : i])\n",
    "        nhpp_temp = torch.FloatTensor(nhpp_temp).unsqueeze(1).unsqueeze(0).cuda()\n",
    "\n",
    "        nhpp = torch.cat((nhpp, nhpp_temp), dim = 2)\n",
    "\n",
    "    else:\n",
    "      nhpp = torch.FloatTensor(nhpp).unsqueeze(1).unsqueeze(0).cuda()\n",
    "      for i in range (0, len(values)):\n",
    "        nhpp_temp = torch.FloatTensor([days[nhpp_nbr], \n",
    "                                       weekdays[nhpp_nbr], \n",
    "                                       months[nhpp_nbr]]).unsqueeze(1).unsqueeze(0).cuda()\n",
    "\n",
    "#         print (nhpp.size(), nhpp_temp.size())\n",
    "        nhpp = torch.cat((nhpp, nhpp_temp), dim = 2)\n",
    "    \n",
    "\n",
    "  return nhpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VNzhoW2umhnb"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CRo6G7R4p167"
   },
   "outputs": [],
   "source": [
    "# The actual network\n",
    "class WaveNet(nn.Module):\n",
    "    def __init__(self, mu=1,n_residue=48, n_skip= 256, dilation_depth=10, \n",
    "                 n_repeat=5, nhpp = 50, nhpp_nbr = 0, additional_features = 0):\n",
    "        # n_residue: residue channels\n",
    "        # n_skip: skip channels\n",
    "        # dilation_depth & n_repeat: dilation layer setup\n",
    "        super(WaveNet, self).__init__()\n",
    "        n_skip = int(np.ceil(n_skip/(dilation_depth*n_repeat))*(dilation_depth*n_repeat))\n",
    "        self.nhpp_nbr = nhpp_nbr\n",
    "        self.dilation_depth = dilation_depth\n",
    "        dilations = self.dilations = [2**i for i in range(dilation_depth)] * n_repeat\n",
    "        self.from_input = nn.Conv1d(in_channels=mu, out_channels=n_residue, kernel_size=1)\n",
    "        self.conv_sigmoid = nn.ModuleList([nn.Conv1d(in_channels=n_residue, out_channels=n_residue, kernel_size=2, dilation=d)\n",
    "                         for d in dilations])\n",
    "        self.conv_tanh = nn.ModuleList([nn.Conv1d(in_channels=n_residue, out_channels=n_residue, kernel_size=2, dilation=d)\n",
    "                         for d in dilations])\n",
    "        self.skip_scale = nn.ModuleList([nn.Conv1d(in_channels=n_residue, out_channels=n_skip, kernel_size=1)\n",
    "                         for d in dilations]) \n",
    "        self.residue_scale = nn.ModuleList([nn.Conv1d(in_channels=n_residue, out_channels=n_residue, kernel_size=1)\n",
    "                         for d in dilations])\n",
    "        self.conv_post_1 = nn.Conv1d(in_channels=n_skip, out_channels=n_skip, kernel_size=1)\n",
    "        \n",
    "        if (additional_features > 0):      \n",
    "          self.linear1 = nn.Linear(in_features= n_skip + nhpp_nbr + additional_features, out_features=mu) # + 3, i.e. month, day, weekday\n",
    "        else:\n",
    "          self.linear1 = nn.Linear(in_features= n_skip + nhpp_nbr, out_features=mu)\n",
    "          \n",
    "          \n",
    "        if (nhpp_nbr + additional_features > 0): \n",
    "          self.linear_nhpp = nn.Linear(in_features = nhpp_nbr + additional_features, \n",
    "                                         out_features = nhpp_nbr + additional_features)\n",
    "          \n",
    "        self.dropout = nn.Dropout (p=0.5)\n",
    "        \n",
    "    def forward(self, input):\n",
    "        # Separating histories\n",
    "        history, nhpp = input\n",
    "        \n",
    "        \n",
    "        \n",
    "        wn_history = self.preprocess(history) # Preparing the history for the Wavenet sequence\n",
    "        ######### Wavenet Sequence ##########\n",
    "        \n",
    "        skip_connections = []\n",
    "        loop_cst = 0\n",
    "        # Convolution sequence\n",
    "        for s, t, skip_scale, residue_scale in zip(self.conv_sigmoid, self.conv_tanh, self.skip_scale, self.residue_scale):\n",
    "            wn_history, skip = self.residue_forward(wn_history, s, t, skip_scale, residue_scale)\n",
    "            skip_connections.append(skip)\n",
    "            loop_cst += 1\n",
    "        # sum up skip connections\n",
    "        wn_history = sum([s[:,:,-wn_history.size(2):] for s in skip_connections]) # If standard Wavenet\n",
    "        ######## End of Wavenet Sequence #########\n",
    "        # \n",
    "        output = self.postprocess(wn_history, nhpp)\n",
    "        \n",
    "        \n",
    "        return output\n",
    "    \n",
    "    def preprocess(self, input):\n",
    "        output = input.unsqueeze(0).unsqueeze(0).cuda()\n",
    "        output = self.from_input(output)\n",
    "        return output\n",
    "    \n",
    "    \n",
    "    def postprocess(self, wn_history, nhpp):\n",
    "        \n",
    "        \n",
    "            \n",
    "        # Discard first nhpp (+1) points as we don't need them - they don't have nhpp history points\n",
    "        wn_history = wn_history[:,:,-wn_history.size(2)+self.nhpp_nbr:]\n",
    "        \n",
    "        start_point = nhpp.size(1) - wn_history.size(2)\n",
    "        \n",
    "        nhpp = nhpp[:,start_point:].unsqueeze(0)\n",
    "        \n",
    "  \n",
    "  \n",
    "        # One layer to process the nhpp\n",
    "        if (add_nhpp_FF == True):\n",
    "          if (nhpp.size(1) + additional_features > 0):\n",
    "            nhpp = torch.tanh(self.linear_nhpp(nhpp.transpose(1,2)).transpose(1,2))\n",
    "        \n",
    "        output = torch.cat((wn_history, nhpp), dim = 1)\n",
    "        \n",
    "        \n",
    "        output = output.transpose(1,2)\n",
    "\n",
    "#         Dropout\n",
    "        if (add_dropout == True):\n",
    "          if (training_so_dropout == True):\n",
    "            output = self.dropout(output)\n",
    "      \n",
    "        output = self.linear1(output).squeeze(0).squeeze(1) # Last layer: linear - for regression\n",
    "        \n",
    "        \n",
    "        return output\n",
    "    \n",
    "    def residue_forward(self, input, conv_sigmoid, conv_tanh, skip_scale, residue_scale):\n",
    "        output = input\n",
    "        output_sigmoid, output_tanh = conv_sigmoid(output), conv_tanh(output)\n",
    "        output = torch.sigmoid(output_sigmoid) * torch.tanh(output_tanh)\n",
    "        skip = skip_scale(output)\n",
    "        output = residue_scale(output)\n",
    "        output = output + input[:,:,-output.size(2):]\n",
    "        return output, skip\n",
    "    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xG1lqPHDGqut"
   },
   "outputs": [],
   "source": [
    "# Function which runs the algorithm\n",
    "\n",
    "def run_algorithm_gpu(nhpp, n_residue, n_skip, dilation_depth, n_repeat,\n",
    "                      max_epoch, batch_size, learn_rate, graph_cst, batches, additional_features = 0,\n",
    "                      dbd_prediction = False, add_nhpp_FF = True, add_dropout = True):\n",
    "    print ('nhpp', nhpp[nhpp_nbr].size(1))\n",
    "    print ('additional features', additional_features)\n",
    "    print ('n_residue', n_residue)\n",
    "    print ('n_skip', n_skip)\n",
    "    print ('dilation_depth', dilation_depth)\n",
    "    print ('n_repeat', n_repeat)\n",
    "    print ('max_epoch', max_epoch)\n",
    "    print ('batch_size', batch_size)\n",
    "    print ('learning rate', learn_rate)\n",
    "    print ('add_nhpp_FF', add_nhpp_FF)\n",
    "    print ('add_dropout', add_dropout)\n",
    "    best_valid_epoch = 0\n",
    "    net = WaveNet(mu=1,n_residue=n_residue,n_skip=n_skip,dilation_depth=dilation_depth,\n",
    "                  n_repeat=n_repeat, nhpp = nhpp, nhpp_nbr = nhpp_nbr,\n",
    "                 additional_features = additional_features).cuda()\n",
    "    optimizer = optim.Adam(net.parameters(),lr=learn_rate)\n",
    "    max_data = len(values)-batch_size\n",
    "    \n",
    "    \n",
    "    loss_fct = RMSELoss()\n",
    "    temp_loss = torch.Tensor(0)\n",
    "\n",
    "    loss_save = [] \n",
    "    valid_loss_save = []\n",
    "    valid_loss = 1000000\n",
    "    last_loss = 1000000\n",
    "    \n",
    "    sequence_lowering_loss = 0\n",
    "    \n",
    "    for epoch in range(max_epoch):\n",
    "        optimizer.zero_grad()\n",
    "        loss = 0\n",
    "        training_so_dropout = True\n",
    "        for j in range(int(batches)):\n",
    "            i = random.randint(0, len(values) - batch_size)\n",
    "            batch = torch.FloatTensor(values[i : i + batch_size]).cuda()\n",
    "\n",
    "            nhpp_sample = nhpp[nhpp_nbr][:,:, i : i + batch_size].cuda()\n",
    "            x = (batch[:-1], nhpp_sample[0,:,:-1])\n",
    "            logits = net(x)\n",
    "            sz = logits.size(0)\n",
    "            temp_loss = loss_fct(logits, batch[-sz:])\n",
    "            \n",
    "            loss = loss + temp_loss\n",
    "         \n",
    "        loss = loss/batches\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        loss_save.append(float(loss))\n",
    "        if (epoch % graph_cst == 0):\n",
    "          training_so_dropout = False\n",
    "      \n",
    "          if (dbd_prediction == False):\n",
    "              batch = torch.FloatTensor(values_valid).cuda()\n",
    "              nhpp_sample = nhpp_valid[nhpp_nbr]\n",
    "\n",
    "              x = (batch[:-1], nhpp_sample[0,:,:-1])\n",
    "\n",
    "              logits = net(x)\n",
    "              l = logits \n",
    "              sz = logits.size(0)\n",
    "              v_loss = loss_fct(logits, batch[-sz:])\n",
    "          else:    \n",
    "            diff, t_loss = test_generation(nt = net, nhpp = nhpp_test[nhpp_nbr], plot=False, prediction_window = 30)\n",
    "            \n",
    "          if (v_loss < valid_loss):\n",
    "            best_valid_epoch = epoch\n",
    "            valid_loss = v_loss\n",
    "            best_net = WaveNet(mu=1,n_residue=n_residue,n_skip=n_skip,dilation_depth=dilation_depth,\n",
    "                                n_repeat=n_repeat, nhpp = nhpp, nhpp_nbr = nhpp_nbr,\n",
    "                                additional_features = additional_features).cuda()\n",
    "            best_net.load_state_dict(copy.deepcopy(net.state_dict()))\n",
    "            \n",
    "            valid_loss_save.append(float(valid_loss))\n",
    "            \n",
    "     \n",
    "    return best_net, loss_save, valid_loss_save, best_valid_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generating each point given its past (used for prediction of 30 points into the future)\n",
    "# Then, generate the next point given its past (including the prediction at the previous step)\n",
    "# Repeat for prediction_window turns.\n",
    "\n",
    "additional_features = 50\n",
    "def test_generation( first_points = 0,last_points = 0, \n",
    "                    prediction_window = 30, nt = None, \n",
    "                    nhpp = None, plot = False):\n",
    "  # Finding the difference between the two \n",
    "  i = 0\n",
    "  loss_fct = RMSELoss()\n",
    "  nhpp_nbr = nhpp.size(1) - additional_features\n",
    "  batch = torch.FloatTensor(values_test[i : (i+batch_size)]).cuda()\n",
    "  nhpp_sample = nhpp[:,:, i : (i+batch_size)].cuda()\n",
    "\n",
    "  x = (batch[:-1], nhpp_sample[0,:,:-1])\n",
    "  \n",
    "  logits = nt(x)\n",
    "  \n",
    "  \n",
    "  start_point = batch.size(0) - logits.size(0)\n",
    "  \n",
    "  \n",
    "  n = len(values_test)-start_point\n",
    "  results = []\n",
    "  for i in range (0,n-prediction_window+1, prediction_window):\n",
    "    index = value2index[ values_test[i + prediction_window] ]\n",
    "    batch_temp = values_test[i : (i + start_point + 1)]\n",
    "    nhpp_temp = nhpp[:,:,i: (i + start_point + 1)].squeeze(0).transpose(0,1)\n",
    "    \n",
    "    for j in range (prediction_window):\n",
    "      \n",
    "      batch = torch.FloatTensor(batch_temp).cuda()\n",
    "      \n",
    "      nhpp_sample = nhpp_temp.clone()\n",
    "      nhpp_sample = nhpp_sample.transpose(0,1).cuda()\n",
    "      x = (batch[:-1], nhpp_sample[:,:-1])\n",
    "      y = nt.forward(x)\n",
    "      batch_temp = batch_temp[1:]\n",
    "      batch_temp.append(torch.FloatTensor([float(y.data)]))\n",
    "  \n",
    "      features = []\n",
    "      \n",
    "      if (additional_features == 50):\n",
    "  \n",
    "        nhpp_temp = nhpp_temp[:][1:]\n",
    "      \n",
    "        days_oh = list(np.zeros(31))\n",
    "        days_oh[days[index]] = 1\n",
    "        features.extend(days_oh)\n",
    "\n",
    "        weekdays_oh = [0, 0, 0, 0, 0, 0, 0]\n",
    "        weekdays_oh[weekdays[index]] = 1\n",
    "        features.extend(weekdays_oh)\n",
    "\n",
    "        months_oh = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
    "        months_oh[months[index]] = 1\n",
    "        features.extend(months_oh)\n",
    "        \n",
    "        if nhpp_nbr > 0:\n",
    "          features.extend(batch_temp[-nhpp_nbr:])\n",
    "\n",
    "        features = torch.Tensor(features).cuda().unsqueeze(0)\n",
    "        nhpp_temp = torch.cat((nhpp_temp, features), dim = 0)\n",
    "        \n",
    "        \n",
    "      else:\n",
    "        nhpp_temp = nhpp_temp[:][1:]\n",
    "        \n",
    "        if nhpp_nbr > 0:\n",
    "          features.extend(batch_temp[-nhpp_nbr:])\n",
    "\n",
    "        features = torch.Tensor(features).cuda().unsqueeze(0)\n",
    "        nhpp_temp = torch.cat((nhpp_temp, features), dim = 0)\n",
    "      \n",
    "        \n",
    "      results.append(float(y.data))\n",
    "  targets2 = values_test[start_point:(start_point+(math.floor(n/prediction_window))*prediction_window)]\n",
    "  \n",
    "  loss2 = loss_fct (torch.FloatTensor(results), torch.FloatTensor(targets2))\n",
    "  \n",
    "  \n",
    "  test_data_outside = (n-prediction_window+1) % prediction_window\n",
    "  \n",
    "  \n",
    "  bench_loss = loss_fct(torch.Tensor(tanke_bench[start_point:(start_point+(math.floor(n/prediction_window))*prediction_window)]), \n",
    "                        torch.Tensor(test_targets_bench[start_point:(start_point+(math.floor(n/prediction_window))*prediction_window)]))\n",
    "  \n",
    "  diff = loss2 - bench_loss\n",
    "  \n",
    "    \n",
    "    \n",
    "  if (plot == True):\n",
    "    print ('generation loss for nhpp = ', nhpp_nbr, ' is:', float(loss2.data.tolist()), 'while benchmark loss for window:', bench_loss)\n",
    "    print ('generation - benchmark', loss2 - bench_loss)\n",
    "    \n",
    "\n",
    "    window_loss = loss_fct(torch.FloatTensor(results[first_points:last_points]), torch.FloatTensor(targets2[first_points:last_points]))\n",
    "    plt.plot(results, ms = 0.1, color = 'green', label = 'predictions')\n",
    "    plt.plot(tanke_bench[start_point:(start_point+(math.floor(n/prediction_window))*prediction_window)], ms = 0.1, label = 'benchmark')\n",
    "    plt.plot(targets2, ms = 0.1, label = 'targets')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "  targets2 = torch.FloatTensor(targets2)\n",
    "  results = torch.FloatTensor(results)\n",
    "  return diff, float(loss2.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cjocfKKBf8sJ"
   },
   "outputs": [],
   "source": [
    "# Applying algorithm to the data, takes a long time\n",
    "\n",
    "additional_features = 50\n",
    "prediction_train_size = 100\n",
    "\n",
    "import time\n",
    "\n",
    "\n",
    "n_skips = [1300]\n",
    "\n",
    "\n",
    "learn_rates = [1e-3]\n",
    "\n",
    "nhpp_nbrs = [0, 20]\n",
    "\n",
    "\n",
    "dilation_depths = [4] \n",
    "\n",
    "training_so_dropout = True\n",
    "add_nhpp_FF = False\n",
    "add_dropout = False\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Constants\n",
    "\n",
    "number_of_experiments = 5 #constant saying how many experiments should be run for one setting of hyperparameters\n",
    "# it's used to calculate the mean standard deviation of the test loss in the case we want to make a comparison with \n",
    "# other models\n",
    "\n",
    "\n",
    "\n",
    "max_epoch = 700\n",
    "batches = 5\n",
    "graph_cst = math.floor(max_epoch/50)\n",
    "print ('graph cst', graph_cst)\n",
    "# batch_size = len(values)\n",
    "n_residue = 25\n",
    "n_repeat = 2\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "calc_length = len(nhpp_nbrs)*len(dilation_depths)*len(n_skips)*len(learn_rates) * number_of_experiments\n",
    "calc_count = 0\n",
    "dbd_prediction = False\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "net = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(int))))\n",
    "nhpp = defaultdict(int)\n",
    "nhpp_test = defaultdict(int)\n",
    "nhpp_valid = defaultdict(int)\n",
    "losses = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(\n",
    "                                lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(int))))))))\n",
    "\n",
    "\n",
    "print (len(values))\n",
    "\n",
    "\n",
    "for nhpp_nbr in nhpp_nbrs:\n",
    "  for learn_rate in learn_rates:\n",
    "    nhpp[nhpp_nbr] = past_points_collector(values = values, nhpp_nbr = nhpp_nbr)\n",
    "    nhpp_test[nhpp_nbr] = past_points_collector(values_test, nhpp_nbr)\n",
    "    nhpp_valid[nhpp_nbr] = past_points_collector(values_valid, nhpp_nbr)\n",
    "\n",
    "    for n in range(len(n_skips)):\n",
    "      \n",
    "      n_skip = n_skips[n] - nhpp_nbr - additional_features\n",
    "\n",
    "        #   for n_skip in n_skips:\n",
    "\n",
    "      for dilation_depth in dilation_depths:\n",
    "            batch_size = 2**(dilation_depth+1) + nhpp_nbr + prediction_train_size\n",
    "            total_train_loss = torch.zeros(max_epoch)\n",
    "            total_valid_loss = []\n",
    "            best_valid_train_loss = []\n",
    "            total_test_loss = []\n",
    "    #     for nhpp_nbr in nhpps:\n",
    "\n",
    "\n",
    "            for repeat in range (number_of_experiments):\n",
    "\n",
    "              \n",
    "              start = time.time()\n",
    "              calc_count += 1\n",
    "\n",
    "\n",
    "              net[nhpp_nbr][n_skip][dilation_depth][learn_rate], loss_save, valid_loss, best_valid_epoch = run_algorithm_gpu(nhpp = nhpp, \n",
    "                                                   n_residue = n_residue, \n",
    "                                                   n_skip = n_skip, \n",
    "                                                   dilation_depth = dilation_depth,  \n",
    "                                                   n_repeat = n_repeat,\n",
    "                                                   batch_size = batch_size, \n",
    "                                                   learn_rate = learn_rate, \n",
    "                                                   max_epoch = max_epoch,\n",
    "                                                   graph_cst = graph_cst,\n",
    "                                                   additional_features = additional_features,\n",
    "                                                   dbd_prediction = dbd_prediction,\n",
    "                                                   batches = batches,\n",
    "                                                   add_nhpp_FF = add_nhpp_FF,\n",
    "                                                   add_dropout = add_dropout)\n",
    "          \n",
    "              print ('best validation epoch', best_valid_epoch)\n",
    "              \n",
    "              \n",
    "              # Calculating test loss\n",
    "              batch = torch.FloatTensor(values_test).cuda()\n",
    "              nhpp_sample = nhpp_test[nhpp_nbr]\n",
    "              x = (batch[:-1], nhpp_sample[0,:,:-1])\n",
    "              logits = net[nhpp_nbr][n_skip][dilation_depth][learn_rate](x)\n",
    "              sz = logits.size(0)\n",
    "              t_loss = loss_fct(logits, batch[-sz:])\n",
    "              \n",
    "              \n",
    "              \n",
    "          \n",
    "              total_train_loss += torch.Tensor(loss_save)\n",
    "              total_valid_loss.append(min(valid_loss))\n",
    "              best_valid_train_loss.append(loss_save[best_valid_epoch])\n",
    "              total_test_loss.append(float(t_loss))\n",
    "              \n",
    "              \n",
    "              \n",
    "              end = time.time()\n",
    "              print ('---------------', 'done', (calc_count/calc_length) * 100, \n",
    "                    '% -----------------', 'time left', (end-start) * (calc_length-calc_count))\n",
    "            \n",
    "            \n",
    "            total_train_loss = total_train_loss / number_of_experiments\n",
    "            total_valid_loss = np.mean(total_valid_loss) \n",
    "            best_valid_train_loss = np.mean(best_valid_train_loss) \n",
    "#             total_test_loss = total_test_loss\n",
    "            \n",
    "            \n",
    "            losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch] = [total_train_loss, \n",
    "                                                                                                                np.mean(total_valid_loss),\n",
    "                                                                                                                np.mean(best_valid_train_loss),\n",
    "                                                                                                                np.mean(total_test_loss),\n",
    "                                                                                                                np.std(total_test_loss)]\n",
    "            \n",
    "            test_generation(first_points = 0, last_points = 60, prediction_window = 30, \n",
    "                            nt = net[nhpp_nbr][n_skip][dilation_depth][learn_rate],\n",
    "                            nhpp = nhpp_test[nhpp_nbr])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tzg822XwpLoz"
   },
   "outputs": [],
   "source": [
    "class l_class:\n",
    "  def __init__(self):\n",
    "\n",
    "        self.nhpp_nbr = 0\n",
    "\n",
    "        self.n_residue  = 0\n",
    "\n",
    "        self.n_skip  = 0\n",
    "\n",
    "        self.n_repeat   = 0\n",
    "        \n",
    "        self.prediction_train_size = 0\n",
    "        \n",
    "        self.dilation_depth = 0\n",
    "        \n",
    "        self.learn_rate = 0\n",
    "        \n",
    "        self.max_epoch = 0\n",
    "        \n",
    "        self.loss_train = []\n",
    "        self.loss_train_zero = []\n",
    "        \n",
    "        self.loss_valid = 0\n",
    "        self.loss_valid_zero = 0\n",
    "        \n",
    "        self.loss_test = 0\n",
    "        self.loss_test_zero = 0\n",
    "        \n",
    "        self.best_valid_train_loss = 0\n",
    "        self.best_valid_train_loss_zero = 0\n",
    "        \n",
    "        self.test_std = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ViWVOynzrkJW"
   },
   "outputs": [],
   "source": [
    "# Showing the results and deciding which ones are good and which ones are not\n",
    "# write_file_title = \"\"\n",
    "write_file_title = \"xgboost_comparison_wavenet_\" + file +  \"_results.txt\"\n",
    "\n",
    "  \n",
    "test_stds = []\n",
    "\n",
    "f = open(write_file_title,\"wb\")\n",
    "object_list = []\n",
    "\n",
    "training_so_dropout = False\n",
    "loss_fct = RMSELoss()\n",
    "min_los = 10000\n",
    "min_diff = 10000\n",
    "los_list = []\n",
    "for nhpp_nbr in losses.keys():\n",
    "  for n_residue in losses[nhpp_nbr].keys():\n",
    "    for n_skip in losses[nhpp_nbr][n_residue].keys():\n",
    "      for dilation_depth in losses[nhpp_nbr][n_residue][n_skip].keys():\n",
    "        for n_repeat in losses[nhpp_nbr][n_residue][n_skip][dilation_depth].keys():\n",
    "          for prediction_train_size in losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat].keys():\n",
    "            for learn_rate in losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat][prediction_train_size].keys():\n",
    "              for max_epoch in losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat][prediction_train_size][learn_rate].keys():\n",
    "                  if( type(losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch]) != int):\n",
    "                    \n",
    "                    # current results\n",
    "                    los_train = losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][0]\n",
    "                    los_valid = losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][1]\n",
    "                    best_valid_train_loss = losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][2]\n",
    "                    test_los = losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][3]\n",
    "                    test_std = losses[nhpp_nbr][n_residue][n_skip][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][4]\n",
    "\n",
    "                    test_stds.append(test_std)\n",
    "                    los_train_zero = losses[0][n_residue][n_skip+nhpp_nbr][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][0]\n",
    "                    los_valid_zero = losses[0][n_residue][n_skip+nhpp_nbr][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][1]\n",
    "                    best_valid_train_loss_zero = losses[0][n_residue][n_skip+nhpp_nbr][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][2]\n",
    "                    test_los_zero = losses[0][n_residue][n_skip+nhpp_nbr][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][3]\n",
    "                    test_std_zero = losses[0][n_residue][n_skip+nhpp_nbr][dilation_depth][n_repeat][prediction_train_size][learn_rate][max_epoch][4]\n",
    "\n",
    "                    print ('nhpp', nhpp_nbr, 'n_skip', n_skip, 'dilation', dilation_depth, \n",
    "                         'valid diff',(los_valid_zero - los_valid)/los_valid_zero,\n",
    "                         'train diff', (min(los_train_zero) - min(los_train))/min(los_train_zero), \n",
    "                        'test loss diff', float((test_los_zero - test_los) / test_los_zero ))\n",
    "                    print ('los valid', los_valid, los_valid_zero)\n",
    "                    print ('test_std and test_std_zero', test_std, test_std_zero)\n",
    "                  \n",
    "\n",
    "    \n",
    "#                     batch = torch.FloatTensor(values_valid).cuda()\n",
    "      \n",
    "#                     nhpp_sample = nhpp_valid[nhpp_nbr]\n",
    "#                     nhpp_sample_zero = nhpp_valid[0]\n",
    "\n",
    "#                     x = (batch[:-1], nhpp_sample[0,:,:-1])\n",
    "#                     x_zero = (batch[:-1], nhpp_sample_zero[0,:,:-1])\n",
    "\n",
    "#                     logits = net[nhpp_nbr][n_skip][dilation_depth][learn_rate](x)\n",
    "#                     logits_zero = net[0][n_skip+nhpp_nbr][dilation_depth][learn_rate](x_zero)\n",
    "              \n",
    "#                     sz = logits.size(0)\n",
    "#                     sz_zero = logits_zero.size(0)\n",
    "                  \n",
    "#                     t_loss = loss_fct(logits, batch[-sz:])\n",
    "#                     t_loss_zero = loss_fct(logits_zero, batch[-sz_zero:])\n",
    "                    \n",
    "#                     plt.plot(batch[-sz:].data.tolist(), label = 'targets')\n",
    "#                     plt.plot(logits.data.tolist(), label = 'non_generated')\n",
    "#                     plt.legend()\n",
    "#                     print (t_loss, t_loss_zero)\n",
    "                    \n",
    "#                     diff, l = test_generation(first_points = 0,last_points = 0, \n",
    "#                     prediction_window = 30, nt = net[nhpp_nbr][n_skip][dilation_depth][learn_rate], \n",
    "#                     nhpp = nhpp_test[nhpp_nbr], plot = True)\n",
    "                    \n",
    "  \n",
    "                    # Writing to file\n",
    "                    object_to_write = l_class()\n",
    "                    \n",
    "                    object_to_write.nhpp_nbr      = nhpp_nbr\n",
    "\n",
    "                    object_to_write.n_residue       = n_residue\n",
    "\n",
    "                    object_to_write.n_skip      = n_skip\n",
    "\n",
    "                    object_to_write.n_repeat   = n_repeat\n",
    "\n",
    "                    object_to_write.prediction_train_size = n_repeat\n",
    "\n",
    "                    object_to_write.learn_rate = learn_rate\n",
    "                    \n",
    "                    object_to_write.dilation_depth = dilation_depth\n",
    "\n",
    "                    object_to_write.max_epoch = max_epoch\n",
    "                    \n",
    "                    object_to_write.loss_train = los_train\n",
    "                    object_to_write.loss_train_zero = los_train_zero\n",
    "                    \n",
    "                    object_to_write.loss_valid = los_valid\n",
    "                    object_to_write.loss_valid_zero = los_valid_zero\n",
    "                    \n",
    "                    object_to_write.loss_test = test_los\n",
    "                    object_to_write.loss_test_zero = test_los_zero\n",
    "\n",
    "                    object_to_write.best_valid_train_loss = best_valid_train_loss\n",
    "                    object_to_write.best_valid_train_loss_zero = best_valid_train_loss_zero\n",
    "                    \n",
    "                    object_to_write.test_std = test_std\n",
    "                    \n",
    "                    \n",
    "                    object_list.append(object_to_write)\n",
    "                    \n",
    "                    \n",
    "#                     plt.show()\n",
    "pickle.dump(object_list, f)\n",
    "plt.hist(test_stds)\n",
    "                    \n",
    "    \n",
    "#                     if (diff < min_diff):\n",
    "#                       min_diff = diff\n",
    "#                       best_nhpp = nhpp_nbr\n",
    "#                       best_n_skip = n_skip\n",
    "#                       best_dilation = dilation_depth\n",
    "    \n",
    "    \n",
    "# #                     plt.plot(los_test, label = str(nhpp_nbr) + ' ' + str(n_skip) + ' ' + str(dilation_depth))       \n",
    "# #                     plt.legend()\n",
    "        \n",
    "#                     los_list.append(l)\n",
    "#                     if (l < min_los):\n",
    "#                       min_los = l\n",
    "#                       best_parameters = [nhpp_nbr, n_skip, dilation_depth, learn_rate]            \n",
    "\n",
    "\n",
    "#                     print ('difference between generated and non generated', loss_fct(torch.Tensor(generated).cuda(), logits))\n",
    "#                     plt.show()\n",
    "  #                   plt.legend()\n",
    "  #                   print (np.mean(los[1400:]), nhpp_nbr)\n",
    "    #                 print (los)\n",
    "    #                 print (nhpp_nbr)\n",
    "f.close()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Thesis_Wavenet.ipynb",
   "provenance": [],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
